{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from urllib import parse\n",
    "from lxml import etree\n",
    "import random\n",
    "import time\n",
    "import math\n",
    "import pandas as pd\n",
    "import json\n",
    "from bs4 import BeautifulSoup\n",
    "import re \n",
    "from requests.packages.urllib3.exceptions import InsecureRequestWarning \n",
    "# 禁用安全请求警告 \n",
    "requests.packages.urllib3.disable_warnings(InsecureRequestWarning)\n",
    "\n",
    "'''\n",
    "获取拉勾网在线的所有城市名称\n",
    "页面解析：在源代码可以直接查看到，故为静态网页，可以直接提取，利用BeatuifulSoup解析返回结果\n",
    "keyword:所要查询关键字，如：'数据分析'\n",
    "'''\n",
    "def get_citys(keyword):\n",
    "    keyword = parse.quote(keyword)\n",
    "    link = \"https://www.lagou.com/jobs/allCity.html?keyword=\"+keyword+\"&px=default&city=%E5%85%A8%E5%9B%BD&positionNum=500+&companyNum=0&isCompanySelected=false&labelWords=\"\n",
    "    headers = {\n",
    "            'User-Agent':'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Mobile Safari/537.36'\n",
    "            }\n",
    "    r = requests.get(link,headers=headers)\n",
    "    citys = []\n",
    "    soup = BeautifulSoup(r.text,'html.parser')\n",
    "    city_word_list = soup.find_all('ul',class_ ='city_list')\n",
    "    for i in range(len(city_word_list)):\n",
    "        city_list = city_word_list[i].find_all('li')\n",
    "        for j in range(len(city_list)):\n",
    "            city = city_list[j].a.text.strip()\n",
    "            citys.append(city)\n",
    "#             print(city)\n",
    "    return citys\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_url(city,keyword,gm):\n",
    "    \n",
    "    '''\n",
    "    编码为URL格式,直接传送的地址非URL格式需要利用urllib.parse进行处理\n",
    "    错误类型：UnicodeEncodeError\n",
    "     'latin-1' codec can't encode characters in position 87-89: ordinal not in range(256)\n",
    "    '''\n",
    "    city = parse.quote(city)\n",
    "    keyword = parse.quote(keyword)\n",
    "    gm = parse.quote(gm)\n",
    "    \n",
    "    url_parse = \"https://www.lagou.com/jobs/positionAjax.json?px=default&city=\"+city+\"&needAddtionalResult=false&gm=\"+gm\n",
    "    url_start = \"https://www.lagou.com/jobs/list_\"+keyword+\"?px=default&gm=\"+ gm + \"&city=\"+city\n",
    "    return url_parse,url_start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#随机选择执行的IP,防止被封\n",
    "def get_proxies():#goole免费代理IP替换或者添加，越多越好\n",
    "    proxie = [\n",
    "        \"134.249.156.3:82\",\n",
    "        \"1.198.72.239:9999\",\n",
    "        \"103.26.245.190:43328\"]\n",
    "    \n",
    "    proxies = {\"http\":str(random.sample(proxie,1))}\n",
    "    return proxies\n",
    "def get_agents():#替换你自己的User-Agent,直接运行不了\n",
    "    agents = [\n",
    "        '**Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3626.121 Safari/537.36',\n",
    "        '**Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3729.157 Mobile Safari/537.36'\n",
    "    ]\n",
    "    agent = random.sample(agents,1)\n",
    "    return agent\n",
    "#定制headers,设置模拟浏览器，反爬\n",
    "def get_headers(url_start,agent):\n",
    "    url_start = url_start\n",
    "    agent = agent\n",
    "    headers = {\n",
    "         #所有数据均来自Network-XHR-headers-Requsts Headers\n",
    "        'Accept': 'application/json, text/javascript, */*; q=0.01',        \n",
    "        'Referer': url_start,\n",
    "        #因为本地的User-Agent已经被封锁，此时有了网上的一个虚拟的，为了反爬可以随机使用几个，类似于下面的proxies\n",
    "#         'Cookie':'user_trace_token=20190515230141-12934d87-ec36-47a4-b6f5-b9fdf6116988; _ga=GA1.2.1677629794.1557932504; _gat=1; LGSID=20190515230142-597afe8c-7722-11e9-99a5-525400f775ce; PRE_UTM=m_cf_cpt_sogou_ztch6; PRE_HOST=; PRE_SITE=; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2Flanding-page%2Fpc%2Fposition2.html%3Futm_source%3Dm_cf_cpt_sogou_ztch6; LGUID=20190515230142-597b0038-7722-11e9-99a5-525400f775ce; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1557932504; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2216abc03944948-07d289ba2ecf5-353166-921600-16abc039450303%22%2C%22%24device_id%22%3A%2216abc03944948-07d289ba2ecf5-353166-921600-16abc039450303%22%7D; sajssdk_2015_cross_new_user=1; _gid=GA1.2.694067267.1557932513; LG_LOGIN_USER_ID=90be76343c3af8869dd6d12563545c28b420c207df6a38cceeb53f0a5d64da34; LG_HAS_LOGIN=1; _putrc=A7C9A275E2B1F6EB123F89F2B170EADC; JSESSIONID=ABAAABAAADEAAFIAD4122C7F738CEEAE9C99959A9756EB3; login=true; unick=%E5%BC%A0%E5%B9%B3; showExpriedIndex=1; showExpriedCompanyHome=1; showExpriedMyPublish=1; hasDeliver=1; gate_login_token=450a1f502d12ad1d9f320d846f06d1ad92603529c81fa8ff8921a53bcf54a82d; index_location_city=%E5%8C%97%E4%BA%AC; X_HTTP_TOKEN=c7dc7114524f2f9f15523975516bfb19cb3eab323b; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1557932554; LGRID=20190515230231-76ee1774-7722-11e9-99a5-525400f775ce; SEARCH_ID=caaf8eb6597d4ecbbe9a2d807d68e78f',\n",
    "        'Host':'www.lagou.com',\n",
    "        'Origin': 'https://www.lagou.com',\n",
    "        'User-Agent':  str(agent),\n",
    "        'X-Anit-Forge-Code': '0',\n",
    "        'X-Anit-Forge-Token': 'None',\n",
    "        'X-Requested-With': 'XMLHttpRequest',\n",
    "        'Connection':'keep_alive'\n",
    "    }  \n",
    "    return headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_json(city,keyword,gm,pn):\n",
    "    #kd是搜索的关键字，若改为“数据分析”即是数据分析相关的数据\n",
    "    data = { 'first': 'true', 'pn': pn, 'kd': keyword }#keyword若是中文，需要为编码的形式\n",
    "    url_parse,url_start = get_url(city,keyword,gm)\n",
    "    agent = get_agents()\n",
    "    headers = get_headers(url_start,agent)\n",
    "    proxies = get_proxies()\n",
    "    print(proxies)\n",
    "    # 请求首页获取cookies \n",
    "    s = requests.Session() \n",
    "    s = s.get(url_start, headers=headers, proxies=proxies,timeout=3)   # 请求首页获取cookies ,allow_redirects=False,verify=False\n",
    "    cookie = s.cookies  # 为此类别获取的cookies \n",
    "    response = requests.post(url_parse,headers=headers,data=data,proxies=proxies,cookies=cookie,timeout=5)\n",
    "    sleep_time = random.randint(4,5)+random.random()\n",
    "    time.sleep(sleep_time)\n",
    "    response.encoding = response.apparent_encoding \n",
    "#     print(response.text)\n",
    "    text = json.loads(response.text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "获取每个城市的全部数据\n",
    "'''\n",
    "def get_city_Pages(city,keyword):\n",
    "    \n",
    "    data = { 'first': 'true', 'pn': 1, 'kd': keyword }#keyword若是中文，需要为编码的形式\n",
    "    city = parse.quote(city)\n",
    "    keyword = parse.quote(keyword)\n",
    "    url_parse = \"https://www.lagou.com/jobs/positionAjax.json?px=default&city=\"+city+\"&needAddtionalResult=false\"\n",
    "    url_start = \"https://www.lagou.com/jobs/list_\"+keyword+\"?px=default&city=\"+city\n",
    "    agent = get_agents()\n",
    "    headers = get_headers(url_start,agent)\n",
    "    proxies = get_proxies()\n",
    "     # 请求首页获取cookies \n",
    "    s = requests.Session() \n",
    "    s = s.get(url_start, headers=headers, proxies=proxies,timeout=3)   # 请求首页获取cookies \n",
    "    cookie = s.cookies  # 为此类别获取的cookies \n",
    "    response = requests.post(url_parse,headers=headers,data=data,proxies=proxies,cookies=cookie,timeout=5)\n",
    "    sleep_time = random.randint(4,5)+random.random()\n",
    "    time.sleep(sleep_time)\n",
    "    response.encoding = response.apparent_encoding \n",
    "#     print(response.text)\n",
    "    text = json.loads(response.text)\n",
    "#     print(text)\n",
    "    totalcount = 0\n",
    "    if 'content' in text:\n",
    "        totalcount = text['content']['positionResult']['totalCount']\n",
    "    else:\n",
    "        print(\"Not exits\")\n",
    "    totalPages = math.ceil(float(totalcount)/15)\n",
    "    city_un = parse.unquote(city)\n",
    "    if totalPages > 0 :\n",
    "        print(\"%s共%s条,可分为%s页\"%(city_un,totalcount,totalPages))\n",
    "#         sleep_time = random.randint(5,15)+random.random()\n",
    "#         time.sleep(sleep_time)#keyerror问题的解决\n",
    "#         print(\"停留%s\"%sleep_time)\n",
    "    else:\n",
    "#         city_un = parse.quote(city)\n",
    "        print(\"%s-----无数据\"%(city_un))\n",
    "#     print(\"共%s条,可分为%s页\"%(totalcount,totalPages))\n",
    "    return totalPages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_totalPages(city,keyword,gm):\n",
    "    city = city\n",
    "    keyword = keyword\n",
    "    totalcount = 0\n",
    "#     pn = 1\n",
    "    text = get_json(city,keyword,gm,'1')\n",
    "    if 'content' in text:\n",
    "        totalcount = text['content']['positionResult']['totalCount']\n",
    "    else:\n",
    "        print(\"Not exits\")\n",
    "    totalPages = math.ceil(float(totalcount)/15)\n",
    "    if totalPages > 0 :\n",
    "        print(\"%s公司规模为%s共%s条,可分为%s页\"%(city,gm,totalcount,totalPages))        \n",
    "#         sleep_time = random.randint(5,15)+random.random()\n",
    "#         time.sleep(sleep_time)#keyerror问题的解决\n",
    "#         print(\"停留%s\"%sleep_time)\n",
    "    else:\n",
    "        print(\"%s公司规模为%s---无数据\"%(city,gm))\n",
    "      \n",
    "    return totalPages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_info(city,keyword,gm,totalPages):\n",
    "    city = city\n",
    "    keyword = keyword\n",
    "    totalPages = totalPages\n",
    "    position_info_all = []#存储数据\n",
    "    for page in range(1,totalPages+1):#从1开始循环因为首页的页数编码是1\n",
    "        time_start = time.time()\n",
    "        text = get_json(city,keyword,gm,page)\n",
    "#         print(text)\n",
    "#         print(proxies)\n",
    "        if 'content'in text:\n",
    "            info = text[\"content\"][\"positionResult\"][\"result\"] \n",
    "            for i in info:\n",
    "                position_info_single = []\n",
    "                position_info_single.append(i.get('positionId','NA'))#职位编号\n",
    "                position_info_single.append(i.get('positionName','NA'))#职位名称\n",
    "                position_info_single.append(i.get('salary','NA'))#薪酬\n",
    "                position_info_single.append(i.get('workYear','NA'))#工作经验\n",
    "                position_info_single.append(i.get('skillLables','NA'))#技能要求\n",
    "                position_info_single.append(i.get('positionAdvantage','NA'))#职位优势\n",
    "                position_info_single.append(i.get('education','NA'))#学历要求\n",
    "                position_info_single.append(i.get('jobNature','NA'))#工作性质\n",
    "                position_info_single.append(i.get('createTime','NA'))#发布时间\n",
    "                position_info_single.append(i.get('companyFullName','NA'))#公司\n",
    "                position_info_single.append(i.get('city','NA'))#城市\n",
    "                position_info_single.append(i.get('companySize','NA'))#公司规模\n",
    "                position_info_single.append(i.get('district','NA'))#区域\n",
    "                position_info_single.append(i.get('financeStage','NA'))#融资情况\n",
    "                position_info_single.append(i.get('firstType','NA'))#公司类别\n",
    "                position_info_single.append(i.get('industryField','NA'))#涉及领域\n",
    "                position_info_single.append(i.get('isSchoolJob','NA'))#是否校招\n",
    "                position_info_single.append(i.get('subwayline','NA'))#地铁\n",
    "                position_info_single.append(i.get('stationname','NA'))#站点\n",
    "                position_info_single.append(i.get('latitude','NA'))#经度\n",
    "                position_info_single.append(i.get('longitude','NA'))#纬度\n",
    "                position_info_all.append(position_info_single) \n",
    "            print(\"第%s页爬取成功,position_info_all现有数据%s行\"%(str(page),str(len(position_info_all))))\n",
    "        else:\n",
    "            print(\"Not exits\")\n",
    "        sleep_time = random.randint(5,15)+random.random()\n",
    "        time.sleep(sleep_time)#keyerror问题的解决\n",
    "        time_end = time.time()\n",
    "        on_time = time_end-time_start\n",
    "        print(\"本页爬行时间%s\"%str(on_time))\n",
    "    return position_info_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "293\n",
      "珠海共3条,可分为1页\n",
      "{'http': \"['218.64.69.79:8080']\"}\n",
      "珠海公司规模为2000人以上共3条,可分为1页\n",
      "{'http': \"['163.125.68.135:8888 ']\"}\n",
      "第1页爬取成功,position_info_all现有数据3行\n",
      "本页爬行时间16.876307725906372\n",
      "珠海的公司规模2000人以上的数据爬完\n",
      "{'http': \"['14.115.104.97:808 ']\"}\n",
      "珠海公司规模为少于15人,15-50人---无数据\n",
      "停留6.7492073387582385\n",
      "{'http': \"['134.249.156.3:82']\"}\n",
      "珠海公司规模为50-150人---无数据\n",
      "停留11.043163736858903\n",
      "{'http': \"['1.198.72.193:9999']\"}\n",
      "Not exits\n",
      "珠海公司规模为150-500人---无数据\n",
      "停留12.172892347874113\n",
      "{'http': \"['183.129.207.86:11206 ']\"}\n",
      "珠海公司规模为500-2000人---无数据\n",
      "停留10.925358249150833\n",
      "position_info_city有3条数据\n",
      "珠海的全部数据爬取成功\n",
      "停留13.576861105003818\n",
      "中山共1条,可分为1页\n",
      "{'http': \"['115.28.148.192:8118']\"}\n",
      "中山公司规模为2000人以上---无数据\n",
      "停留9.102117120811412\n",
      "{'http': \"['134.249.156.3:82']\"}\n",
      "中山公司规模为少于15人,15-50人---无数据\n",
      "停留7.852819219941637\n",
      "{'http': \"['1.192.243.166:9999 ']\"}\n",
      "中山公司规模为50-150人---无数据\n",
      "停留15.585111278379154\n",
      "{'http': \"['163.125.68.135:8888 ']\"}\n",
      "中山公司规模为150-500人共1条,可分为1页\n",
      "{'http': \"['171.80.2.110:9999']\"}\n",
      "第1页爬取成功,position_info_all现有数据1行\n",
      "本页爬行时间20.245324850082397\n",
      "中山的公司规模150-500人的数据爬完\n",
      "{'http': \"['218.64.69.79:8080']\"}\n",
      "中山公司规模为500-2000人---无数据\n",
      "停留12.88375543553882\n",
      "position_info_city有4条数据\n",
      "中山的全部数据爬取成功\n",
      "停留12.943937855541314\n",
      "镇江-----无数据\n",
      "停留9.276375587886204\n",
      "湛江-----无数据\n",
      "停留8.685334392703105\n",
      "株洲共1条,可分为1页\n",
      "{'http': \"['120.83.108.77:9999 ']\"}\n",
      "株洲公司规模为2000人以上---无数据\n",
      "停留5.101241187083019\n",
      "{'http': \"['122.193.244.243:9999 ']\"}\n",
      "株洲公司规模为少于15人,15-50人共1条,可分为1页\n",
      "{'http': \"['14.115.104.97:808 ']\"}\n",
      "Not exits\n",
      "本页爬行时间16.907893180847168\n",
      "株洲的公司规模少于15人,15-50人的数据爬完\n",
      "{'http': \"['1.192.243.166:9999 ']\"}\n",
      "株洲公司规模为50-150人---无数据\n",
      "停留7.160573629384684\n",
      "{'http': \"['115.28.148.192:8118']\"}\n",
      "株洲公司规模为150-500人---无数据\n",
      "停留7.411860943214564\n",
      "{'http': \"['1.198.72.239:9999']\"}\n",
      "株洲公司规模为500-2000人---无数据\n",
      "停留7.505609365573439\n",
      "position_info_city有4条数据\n",
      "株洲的全部数据爬取成功\n",
      "停留12.079427715518644\n",
      "淄博-----无数据\n",
      "停留5.942007353497083\n",
      "肇庆-----无数据\n",
      "停留8.137823183299782\n",
      "张家口-----无数据\n",
      "停留13.28549592337901\n",
      "漳州-----无数据\n",
      "停留14.207343278491953\n",
      "遵义-----无数据\n",
      "停留14.95192348923606\n",
      "驻马店-----无数据\n",
      "停留11.949976561928546\n",
      "长治-----无数据\n",
      "停留15.335753043318482\n",
      "枣庄-----无数据\n",
      "停留14.370120856259682\n",
      "资阳-----无数据\n",
      "停留6.685431142101895\n",
      "舟山-----无数据\n",
      "停留8.37063967450687\n",
      "自贡-----无数据\n",
      "停留7.527898435115426\n",
      "周口共1条,可分为1页\n",
      "{'http': \"['112.85.169.126:9999']\"}\n",
      "周口公司规模为2000人以上---无数据\n",
      "停留6.014899216823151\n",
      "{'http': \"['1.197.204.217:9999 ']\"}\n",
      "周口公司规模为少于15人,15-50人---无数据\n",
      "停留13.197189074964884\n",
      "{'http': \"['112.85.164.68:9999']\"}\n",
      "周口公司规模为50-150人共1条,可分为1页\n",
      "{'http': \"['1.198.72.239:9999']\"}\n",
      "第1页爬取成功,position_info_all现有数据1行\n",
      "本页爬行时间19.017383813858032\n",
      "周口的公司规模50-150人的数据爬完\n",
      "{'http': \"['120.83.108.77:9999 ']\"}\n",
      "周口公司规模为150-500人---无数据\n",
      "停留14.04838082143874\n",
      "{'http': \"['103.26.245.190:43328']\"}\n",
      "周口公司规模为500-2000人---无数据\n",
      "停留9.220707758209855\n",
      "position_info_city有5条数据\n",
      "周口的全部数据爬取成功\n",
      "停留5.143233270941914\n",
      "昭通-----无数据\n",
      "停留7.923482816445571\n",
      "张掖-----无数据\n",
      "停留13.237375799756679\n",
      "张家界-----无数据\n",
      "停留11.974681287128595\n",
      "中卫-----无数据\n",
      "停留7.505663442219061\n",
      "全部数据存储成功（CSV格式）----持续奔跑中！\n",
      "[[5858185, '数据分析师', '15k-30k', '不限', ['数据挖掘', '数据架构', 'MySQL', '数据分析'], '免费三餐，数据分析', '不限', '全职', '2019-04-29 17:45:35', '北京金山软件有限公司', '珠海', '2000人以上', '香洲区', '上市公司', '开发|测试|运维类', '移动互联网', 0, None, None, '22.34793', '113.601205'], [1978586, '数据分析师', '1k-2k', '应届毕业生', [], '提供岗位培训，表现优异者可转正', '不限', '实习', '2019-05-20 16:25:19', '成都西山居互动娱乐科技有限公司珠海分公司', '珠海', '2000人以上', '香洲区', '不需要融资', '产品|需求|项目类', '游戏', 1, None, None, '22.348471', '113.601247'], [5945889, '高级数据分析师', '18k-30k', '3-5年', ['数据分析', '数据挖掘', '数据处理'], '团队优秀 业务前景好 六险一金 年度旅行', '本科', '全职', '2019-05-22 17:37:33', '北京金山软件有限公司', '珠海', '2000人以上', '香洲区', '上市公司', '开发|测试|运维类', '移动互联网', 0, None, None, '22.347929', '113.601205'], [5339042, '数据分析实习生（中山）', '2k-3k', '应届毕业生', ['数据分析', 'MySQL', '数据处理'], '提供转正，周末双休，专题培训，法定假日', '本科', '实习', '2019-05-22 09:34:00', '广东精点数据科技股份有限公司', '中山', '150-500人', '中山市市辖区', '不需要融资', '开发|测试|运维类', '移动互联网,数据服务', 1, None, None, '22.503981', '113.403275'], [5784906, '数据分析工程师', '4k-5k', '1-3年', [], '员工旅游、补充医疗保险、绩效奖金、餐补', '大专', '全职', '2019-05-22 17:22:48', '中科三清科技有限公司', '周口', '50-150人', '川汇区', '不需要融资', '产品|需求|项目类', '其他,移动互联网', 0, None, None, '33.61007', '114.669126']]\n"
     ]
    }
   ],
   "source": [
    "def main():\n",
    "    #公司规模\n",
    "    gms = ['2000人以上','少于15人,15-50人','50-150人','150-500人','500-2000人']    \n",
    "    keyword = '数据分析'\n",
    "    citys = get_citys(keyword)\n",
    "    position_info_all = []\n",
    "    columns = ['职位编号','职位名称','薪酬','工作经验','技能要求','职位优势','学历要求','工作形式','发布时间',\n",
    "               '公司','城市','规模','区域','融资情况','公司类别','涉及领域','是否校招',\n",
    "              '地铁','站点','经度','纬度']\n",
    "    print(len(citys))\n",
    "    for i in range(0,len(citys)):\n",
    "        position_info_city = []\n",
    "        city = citys[i]\n",
    "        cityPages = get_city_Pages(city,keyword)\n",
    "        if cityPages > 0 :\n",
    "            for j in range(len(gms)):\n",
    "                gm = gms[j]\n",
    "                totalPages = get_totalPages(city,keyword,gm)\n",
    "                if totalPages>0:\n",
    "                    \n",
    "                    position_info_city_gm = get_info(city,keyword,gm,totalPages)\n",
    "                    for m in range(len(position_info_city_gm)):\n",
    "                        item = position_info_city_gm[m]\n",
    "                        position_info_city.append(item)\n",
    "                        position_info_all.append(item)\n",
    "    #                 print(position_info_all)\n",
    "                    print(\"%s的公司规模%s的数据爬完\"%(city,gm))\n",
    "                else:\n",
    "    #                 print(\"%s的公司规模%s无数据\"%(city,gm))\n",
    "                    sleep_time = random.randint(5,15)+random.random()\n",
    "                    time.sleep(sleep_time)#keyerror问题的解决\n",
    "                    print(\"停留%s\"%sleep_time)\n",
    "            print(\"position_info_all有%s条数据\"%len(position_info_all))\n",
    "            print(\"%s的全部数据爬取成功\"%city)\n",
    "            sleep_time = random.randint(5,15)+random.random()\n",
    "            time.sleep(sleep_time)#keyerror问题的解决\n",
    "            print(\"停留%s\"%sleep_time)\n",
    "            df_city = pd.DataFrame(data = position_info_city,columns =columns )\n",
    "            df_city.to_csv('%s.csv'%(city),index = False,encoding=\"utf_8_sig\")\n",
    "        else:\n",
    "            sleep_time = random.randint(5,15)+random.random()\n",
    "            time.sleep(sleep_time)#keyerror问题的解决\n",
    "            print(\"停留%s\"%sleep_time)\n",
    "#         position_info_all.append(position_info_city)\n",
    "    df_all = pd.DataFrame(data=position_info_all,columns=columns)\n",
    "    df_all.to_csv('all_city.csv',index = False,encoding=\"utf_8_sig\")\n",
    "    print(\"全部数据存储成功（CSV格式）----持续奔跑中！\") \n",
    "    print(position_info_all)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
